scirius源码分析

scirius

scirius介绍

scirius为suricata提供了一个web，用来管理规则以及处理告警，我们分析一下他是如何实现的。

先分析主目录的views.py

定义了重定向到kinaba、evebox、moloch的路由，没什么关键函数。我们主要关心的是hunt和manage。

rules

看一下rules目录下的views.py

访问manage页面，主要实现了两个功能

• suricata状态监控，配置
• rule管理
  我们看一下是如何实现的？
  抓包发现前端会不断请求以下路径

用来获取suricata状态的info路径对应函数如下

rules/views.py

PROBE = __import__(settings.RULESET_MIDDLEWARE)
def info(request):
    data = {'status': 'green'}
    if request.GET.__contains__('query'):
        info = PROBE.common.Info()
        query = request.GET.get('query', 'status')
        if query == 'status':
            data = {'running': info.status()}
        elif query == 'disk':
            data = info.disk()
        elif query == 'memory':
            data = info.memory()
        elif query == 'used_memory':
            data = info.used_memory()
        elif query == 'cpu':
            data = info.cpu()
    return JsonResponse(data, safe=False)

#在settings.py里可以看到RULESET_MIDDLEWARE='suricata',所以PROBE相当于__import__(settings.suricata)，动态加载了suricata。而suricata/common.py如下
import psutil
from rest_framework import serializers

from django.conf import settings


if settings.SURICATA_UNIX_SOCKET:
    try:
        import suricatasc
    except:
        settings.SURICATA_UNIX_SOCKET = None


class Info():
    def status(self):
        suri_running = 'danger'
        # 判断suricata是否运行则有两种方式,如果配置了suricata_unix_socket则调用suricatasc并发送uptime来判断。
        if settings.SURICATA_UNIX_SOCKET:
            sc = suricatasc.SuricataSC(settings.SURICATA_UNIX_SOCKET)
            try:
                sc.connect()
            except:
                return 'danger'
            res = sc.send_command('uptime', None)
            if res['return'] == 'OK':
                suri_running = 'success'
            sc.close()
        # 如果没有配置SURICATA_UNIX_SOCKET则调用psutil.process_iter()来判断是否存在Suricata-Main来实现
        else:
            for proc in psutil.process_iter():
                try:
                    pinfo = proc.as_dict(attrs=['name'])
                except psutil.NoSuchProcess:
                    pass
                else:
                    if pinfo['name'] == 'Suricata-Main':
                        suri_running = 'success'
                        break
        return suri_running
#可以看到该函数调用psutil获取cpu/disk/mem状态
    def disk(self):
        return psutil.disk_usage('/')

    def memory(self):
        return psutil.virtual_memory()

    def used_memory(self):
        mem = psutil.virtual_memory()
        return round(mem.used * 100. / mem.total, 1)

    def cpu(self):
        return psutil.cpu_percent(interval=0.2)

suricatasc可以实时和suricata进程进行交互

接下来看一下核心功能之一-规则的管理，先大概了解一下功能

• /rules/source可以看到规则源
• /rules/source/add_public可以增加公共规则源
• /rules/source/add可以编辑自定义规则源
• /rules/ruleset/add可以新增规则集
• /rules/ruleset/n/update可以更新规则集
• /rules/ruleset/n/copy可以复制规则集
• /rules/ruleset/n/delete可以删除规则集
• /rules/ruleset/n/edit可以编辑规则集
• /rules/ruleset/1/edit?mode=sources可以启用/禁用source
• /rules/ruleset/1/edit?mode=categories可以启用/禁用categories
• /rules/ruleset/1/addsupprule可以禁用某条rule
• /rules/ruleset/1/edit?mode=rules可以恢复某条被禁用的rule
• /rules/category/n/可以查看category
• /rules/category/n/enable可以启用category
• /rules/category/n/disable可以禁用category
• /rules/category/n/transform
• /rules/rule/pk/n/可以查看rule的定义
• /rules/rule/pk/n/disable可以禁用rule
• /rules/rule/pk/n/enable可以启用rule
• /rules/rule/n/edit
• /rules/rule/n/disable
• /rules/rule/n/enable
• /rules/rule/n/threshold?action=threshold为规则增加触发的阀值
• /rules/rule/n/threshold?action=suppress过滤某ip触发改规则

  页面如下

/rules/ruleset 列出ruleset规则集

/rules/ruleset页面代码如下

# Create your views here.
def index(request):
    # Ruleset.objects.all()获取已创建的ruleset，并返回给index.html模板
    ruleset_list = Ruleset.objects.all().order_by('-created_date')[:5]
    source_list = Source.objects.all().order_by('-created_date')[:5]
    context = {'ruleset_list': ruleset_list,
               'source_list': source_list}
    try:
        context['probes'] = ['"' + x + '"' for x in PROBE.models.get_probe_hostnames()]
    except:
        pass
    return scirius_render(request, 'rules/index.html', context)

页面如下

/rules/source 列出规则源

/rules/source页面代码如下

def sources(request):
    from scirius.utils import get_middleware_module
    # 调用scirius.utils里的get_middleware_module函数
    sources = get_middleware_module('common').get_sources().order_by('name')

    for source_ in sources:
        if source_.cats_count == 0:
            source_.build_counters()

    context = {'sources': sources}
    print(context)
    return scirius_render(request, 'rules/sources.html', context)
# 可以看到get_middleware_module用来import_module
def get_middleware_module(module):
    return import_module('%s.%s' % (settings.RULESET_MIDDLEWARE, module))
# setting中RULESET_MIDDLEWARE值为suricata
RULESET_MIDDLEWARE = 'suricata'
# suricata下common.py里get_sources如下
def get_sources():
    from rules.models import Source
    return Source.objects.all()

最后获取了Source.objects.all()

页面如下

/rules/source/add_public增加公共规则源

/rules/source/add_public页面代码及分析如下

def add_public_source(request):
    # 鉴权
    if not request.user.is_staff:
        return scirius_render(request, 'rules/add_public_source.html', {'error': 'Unsufficient permissions'})

    try:
    # 调用get_public_sources()
        public_sources = get_public_sources()
    except Exception as e:
        return scirius_render(request, 'rules/add_public_source.html', {'error': e})

    if request.is_ajax():
        return JsonResponse(public_sources['sources'])
    # 当enable一个public source时,会post一个forms
    if request.method == 'POST':
        # 会创建一个AddPublicSourceForm实例
        form = AddPublicSourceForm(request.POST)
        if form.is_valid():
            source_id = form.cleaned_data['source_id']
            source = public_sources['sources'][source_id]
            source_uri = source['url']
            params = {"__version__": "5.0"}
            if 'secret_code' in form.cleaned_data:
                params.update({'secret-code': form.cleaned_data['secret_code']})
            # uri与secret_code拼接
            source_uri = source_uri % params
            # 数据库Source表中增加一个对象,Source.objects.create时会触发很多其他的函数
            try:
                src = Source.objects.create(
                    name=form.cleaned_data['name'],
                    uri=source_uri,
                    method='http',
                    created_date=timezone.now(),
                    datatype=source['datatype'],
                    cert_verif=True,
                    public_source=source_id,
                    use_iprep=form.cleaned_data['use_iprep']
                )
            except IntegrityError as error:
                return scirius_render(request, 'rules/add_public_source.html', {'form': form, 'error': error})
            # 如果指定了ruleset,则更新ruleset增加一个源
            try:
                ruleset_list = form.cleaned_data['rulesets']
            except:
                ruleset_list = []
            rulesets = [ruleset.pk for ruleset in ruleset_list]
            if len(ruleset_list):
                for ruleset in ruleset_list:
                    UserAction.create(
                        action_type='create_source',
                        comment=form.cleaned_data['comment'],
                        user=request.user,
                        source=src,
                        ruleset=ruleset
                    )
            else:
                UserAction.create(
                    action_type='create_source',
                    comment=form.cleaned_data['comment'],
                    user=request.user,
                    source=src,
                    ruleset='No Ruleset'
                )
            ruleset_list = ['"' + ruleset.name + '"' for ruleset in ruleset_list]
            # 最后返回add_public_source.html模板,而add_public_source.html模板中会判断update是否为True,如果为True则include "rules/import_and_add_source.html",在import_and_add_source.html中会调用update_activate_source这个函数,在后面看一下这个函数的功能
            return scirius_render(
                request,
                'rules/add_public_source.html',
                {'source': src, 'update': True, 'rulesets': rulesets, 'ruleset_list': ruleset_list}
            )
        else:
            return scirius_render(
                request,
                'rules/add_public_source.html',
                {'form': form, 'error': 'form is not valid'}
            )
    # 如果不是post请求,则会把source.yaml中解析出来的sources和Ruleset.objects.all()作为参数传递给add_public_source.html模板
    rulesets = Ruleset.objects.all()
    return scirius_render(
        request,
        'rules/add_public_source.html',
        {'sources': public_sources['sources'], 'rulesets': rulesets}
    )


def get_public_sources(force_fetch=True):
    # 路径拼接，找到git-sources文件夹下的sources.yaml
    sources_yaml = os.path.join(settings.GIT_SOURCES_BASE_DIRECTORY, 'sources.yaml')
    # 如果sources.yaml不存在则调用fetch_public_sources()
    if not os.path.exists(sources_yaml) or force_fetch is True:
        try:
            fetch_public_sources()
        except Exception as e:
            raise Exception(e)

    public_sources = None
    # 读取sources.yaml文件
    with open(sources_yaml, 'r', encoding='utf-8') as stream:
        buf = stream.read()
        # replace dash by underscode in keys
        yaml_data = re.sub(r'(\s+\w+)-(\w+):', r'\1_\2:', buf)
        # FIXME error handling
        
        public_sources = yaml.safe_load(yaml_data)

    if public_sources['version'] != 1:
        raise Exception("Unsupported version of sources definition")

    # get list of already defined public sources
    # 获取已经定义了的public sources
    defined_pub_source = Source.objects.exclude(public_source__isnull=True)
    # 列表表达式获取已经存在在数据库里的源
    added_sources = [x.public_source for x in defined_pub_source]

    for source_ in public_sources['sources']:
        if 'support_url' in public_sources['sources'][source_]:
            public_sources['sources'][source_]['support_url_cleaned'] = public_sources['sources'][source_]['support_url'].split(' ')[0]
        if 'subscribe_url' in public_sources['sources'][source_]:
            public_sources['sources'][source_]['subscribe_url_cleaned'] = public_sources['sources'][source_]['subscribe_url'].split(' ')[0]
        if public_sources['sources'][source_]['url'].endswith('.rules'):
            public_sources['sources'][source_]['datatype'] = 'sig'
        elif public_sources['sources'][source_]['url'].endswith('z'):
            public_sources['sources'][source_]['datatype'] = 'sigs'
        else:
            public_sources['sources'][source_]['datatype'] = 'other'
        #这里做了判断，如果这个源已经在Source里了，则设置added标签为True
        if source_ in added_sources:
            public_sources['sources'][source_]['added'] = True
        else:
            public_sources['sources'][source_]['added'] = False

    return public_sources


def fetch_public_sources():
    #翻看rules的model可以看到SystemSettings这个class,其中有use_http_proxy，http_proxy，https_proxy,use_elasticsearch,custom_elasticsearch,elasticsearch_url这几个字段，而get_proxy_params这个函数是用来获取http/https proxy的,但是程序没有直接调用SystemSettings,而是调用get_system_settings(),在这个函数里面会去判断SystemSettings.objects.all()是否存在,如果不存在则创建一个默认的配置
    proxy_params = get_system_settings().get_proxy_params()
    try:
        hdrs = {'User-Agent': 'scirius'}
        # 可以看到程序会从https://www.openinfosecfoundation.org/rules/index.yaml这个网站去获取sources.yaml
        if proxy_params:
            resp = requests.get(settings.DEFAULT_SOURCE_INDEX_URL, proxies=proxy_params, headers=hdrs)
        else:
            resp = requests.get(settings.DEFAULT_SOURCE_INDEX_URL, headers=hdrs)
        resp.raise_for_status()
    except requests.exceptions.ConnectionError as e:
        if "Name or service not known" in str(e):
            raise IOError("Connection error 'Name or service not known'")
        elif "Connection timed out" in str(e):
            raise IOError("Connection error 'Connection timed out'")
        else:
            raise IOError("Connection error '%s'" % (e))
    except requests.exceptions.HTTPError:
        if resp.status_code == 404:
            raise IOError("URL not found on server (error 404), please check URL")
        raise IOError("HTTP error %d sent by server, please check URL or server" % (resp.status_code))
    except requests.exceptions.Timeout:
        raise IOError("Request timeout, server may be down")
    except requests.exceptions.TooManyRedirects:
        raise IOError("Too many redirects, server may be broken")
    # 把获取到的yaml文件复制成yaml
    # store as sources.yaml
    if not os.path.isdir(settings.GIT_SOURCES_BASE_DIRECTORY):
        os.makedirs(settings.GIT_SOURCES_BASE_DIRECTORY)
    sources_yaml = os.path.join(settings.GIT_SOURCES_BASE_DIRECTORY, 'sources.yaml')
    with open(sources_yaml, 'wb') as sfile:
        sfile.write(resp.content)


def update_source(request, source_id):
    src = get_object_or_404(Source, pk=source_id)
    if not request.user.is_staff:
        return redirect(src)

    if request.method != 'POST':  # If the form has been submitted...
        if request.is_ajax():
            data = {}
            data['status'] = False
            data['errors'] = "Invalid method for page"
            return JsonResponse(data)
        return source(request, source_id, error="Invalid method for page")
    try:
        if hasattr(PROBE.common, 'update_source'):
        
            return PROBE.common.update_source(request, src)
        # 调用Source的update方法,判断是否是first import 如果是的话就设置firstimport = True具体函数可以去models里看。
        src.update()
    # 如果update失败则返回error和status为false
    except Exception as errors:
        if request.is_ajax():
            data = {}
            data['status'] = False
            data['errors'] = str(errors)
            return JsonResponse(data)
        if isinstance(errors, (IOError, OSError)):
            _msg = 'Can not fetch data'
        elif isinstance(errors, ValidationError):
            _msg = 'Source is invalid'
        elif isinstance(errors, SuspiciousOperation):
            _msg = 'Source is not correct'
        else:
            _msg = 'Error updating source'
        msg = '%s: %s' % (_msg, errors)
        return source(request, source_id, error=msg)

    if request.is_ajax():
        data = {}
        data['status'] = True
        data['redirect'] = True
        return JsonResponse(data)
    supdate = SourceUpdate.objects.filter(source=src).order_by('-created_date')
    if len(supdate) == 0:
        return redirect(src)

    return redirect('changelog_source', source_id=source_id)


def test_source(request, source_id):
    print("test_source")
    source = get_object_or_404(Source, pk=source_id)
    # models里test函数如下
    sourceatversion = get_object_or_404(SourceAtVersion, source=source, version='HEAD')
    return JsonResponse(sourceatversion.test())


#models.py
class Source(models.Model):
    TMP_DIR = "/tmp/"
    @transaction.atomic
    def update(self):
        # look for categories list: if none, first import
        categories = Category.objects.filter(source=self)
        firstimport = False
        if not categories:
            firstimport = True

        if self.method not in ['http', 'local']:
            raise FieldError("Currently unsupported method")


    def __init__(self, *args, **kwargs):
        models.Model.__init__(self, *args, **kwargs)
        if (self.method == 'http'):
            self.update_ruleset = self.update_ruleset_http
        else:
            self.update_ruleset = None
        self.first_run = False
        self.updated_rules = {"added": [], "deleted": [], "updated": []}
        if len(Flowbit.objects.filter(source=self)) == 0:
            self.init_flowbits = True
        else:
            self.init_flowbits = False

        from scirius.utils import get_middleware_module
        self.custom_data_type = get_middleware_module('common').custom_source_datatype()

        need_update = False
        #调用update_ruleset,update_ruleset=update_ruleset_http
        if self.update_ruleset:
            # 生成规则的临时文件
            f = tempfile.NamedTemporaryFile(dir=self.TMP_DIR)
            need_update = self.update_ruleset(f)

            if need_update:
                # 调用不同的函数来处理规则文件,
                if self.datatype == 'sigs':
                    self.handle_rules_in_tar(f)
                elif self.datatype == 'sig':
                    # handle_rules_file里面会调用get_categories,借着会调用get_rules,如果rule不存在则新建rule
                    self.handle_rules_file(f)
                elif self.datatype == 'other':
                    self.handle_other_file(f)
                elif self.datatype == 'b64dataset':
                    self.handle_b64dataset(f)

                if self.datatype in self.custom_data_type:
                    self.handle_custom_file(f)

        if need_update:
            if self.datatype in ('sig', 'sigs') and not firstimport:
                #调用create_update()记录更新的内容
                self.create_update()

            if self.datatype in self.custom_data_type:
                from scirius.utils import get_middleware_module
                source_path = os.path.join(settings.GIT_SOURCES_BASE_DIRECTORY, str(self.pk), 'rules')
                get_middleware_module('common').update_custom_source(source_path)

            for rule in self.updated_rules["deleted"]:
                rule.delete()
            self.needs_test()


    def handle_rules_file(self, f):
        f.seek(0)
        if (tarfile.is_tarfile(f.name)):
            raise OSError("This is a tar file and not a individual signature file, please select another category")
        f.seek(0)

        self.updated_date = timezone.now()
        self.first_run = False
        repo = self.get_git_repo(delete=True)
        rules_dir = os.path.join(settings.GIT_SOURCES_BASE_DIRECTORY, str(self.pk), 'rules')

        # create rules dir if needed
        if not os.path.isdir(rules_dir):
            os.makedirs(rules_dir)

        # copy file content to target
        f.seek(0)
        os.fsync(f)
        shutil.copy(f.name, os.path.join(rules_dir, 'sigs.rules'))

        index = repo.index
        if len(index.diff(None)) or self.first_run:
            os.environ['USERNAME'] = 'scirius'
            index.add(["rules"])
            message = 'source version at %s' % (self.updated_date)
            index.commit(message)

        self.save()

        # Now we must update SourceAtVersion for this source
        # or create it if needed
        # 更新source ersion
        self.create_sourceatversion()
        # category based on filename
        category = Category.objects.filter(source=self, name=('%s Sigs' % (self.name))[:100])
        if not category:
            # 如果category不存在,则新建category
            category = Category.objects.create(
                source=self,
                name=('%s Sigs' % (self.name))[:100],
                created_date=timezone.now(),
                filename=os.path.join('rules', 'sigs.rules')
            )
            # 调用get_rules函数
            category.get_rules(self)
        else:
            category = category[0]

        category.get_rules(self)
        if len(Rule.objects.filter(category=category)) == 0:
            category.delete()
            raise ValidationError('The source %s contains no valid signature' % self.name)


    def get_rules(self, source, existing_rules_hash=None):
        # parse file
        # return an object with updates
        getsid = re.compile(r"sid *: *(\d+)")
        getrev = re.compile(r"rev *: *(\d+)")
        getmsg = re.compile(r"msg *: *\"(.*?)\"")
        source_git_dir = os.path.join(settings.GIT_SOURCES_BASE_DIRECTORY, str(self.source.pk))
        rfile = open(os.path.join(source_git_dir, self.filename))

        rules_update = {"added": [], "deleted": [], "updated": []}
        rules_unchanged = []
        # existing_rules_hash字典，把所有sid和rule对象对应起来
        if existing_rules_hash is None:
            existing_rules_hash = {}
            for rule in Rule.objects.all().prefetch_related('category'):
                existing_rules_hash[rule.sid] = rule

        rules_list = []
        # 把指定category的rule给遍历出来
        for rule in Rule.objects.filter(category=self):
            rules_list.append(rule)

        flowbits = {'added': {'flowbit': [], 'through_set': [], 'through_isset': []}}
        existing_flowbits = Flowbit.objects.all().order_by('-pk')
        if len(existing_flowbits):
            flowbits['last_pk'] = existing_flowbits[0].pk
        else:
            flowbits['last_pk'] = 1
        for key in ('flowbits', 'hostbits', 'xbits'):
            flowbits[key] = {}
            for flowb in Flowbit.objects.filter(source=source, type=key):
                flowbits[key][flowb.name] = flowb

        creation_date = timezone.now()

        rules_groups = {}
        if source.use_iprep:
            rules_groups = self.build_sigs_group()

        with transaction.atomic():
            duplicate_source = set()
            duplicate_sids = set()
            # 打开rule文件,并通过正则找出rev,sid,msg字段
            for line in rfile.readlines():
                state = True
                if line.startswith('#'):
                    # check if it is a commented signature
                    if "->" in line and "sid" in line and ")" in line:
                        line = line.lstrip("# ")
                        state = False
                    else:
                        continue
                match = getsid.search(line)
                if not match:
                    continue
                sid = match.groups()[0]
                match = getrev.search(line)
                if match:
                    rev = int(match.groups()[0])
                else:
                    rev = None
                match = getmsg.search(line)
                if not match:
                    msg = ""
                else:
                    msg = match.groups()[0]
                # 调用add_group_signature函数处理每一行规则,而该函数调用rule_idstools.parse来解析规则
                if source.use_iprep and Rule.GROUPSNAMEREGEXP.match(msg):
                    self.add_group_signature(rules_groups, line, existing_rules_hash, source, flowbits, rules_update, rules_unchanged)
                else:
                    if int(sid) in existing_rules_hash:
                        # FIXME update references if needed
                        rule = existing_rules_hash[int(sid)]
                        if rule.category.source != source:
                            source_name = rule.category.source.name
                            duplicate_source.add(source_name)
                            duplicate_sids.add(sid)
                            if len(duplicate_sids) == 20:
                                break
                            continue
                        if rev is None or rule.rev < rev or rule.group is True:
                            rule.content = line
                            if rev is None:
                                rule.rev = 0
                            else:
                                rule.rev = rev
                            if rule.category != self:
                                rule.category = self
                            rule.msg = msg
                            rules_update["updated"].append(rule)
                            rule.updated_date = creation_date
                            # 这个函数会利用rule_idstools的parse函数解析rule.content,并且为rule.sid,rule.rev.rule.msg等属性赋值
                            rule.parse_metadata()
                            # 最后save
                            rule.save()
                            rule.parse_flowbits(source, flowbits)
                        else:
                            rules_unchanged.append(rule)
                    else:
                        if rev is None:
                            rev = 0
                        rule = Rule(
                            category=self,
                            sid=sid,
                            rev=rev,
                            content=line,
                            msg=msg,
                            state_in_source=state,
                            state=state,
                            imported_date=creation_date,
                            updated_date=creation_date
                        )
                        rule.parse_metadata()
                        rules_update["added"].append(rule)
                        rule.parse_flowbits(source, flowbits, addition=True)

            if len(duplicate_sids):
                sids = sorted(duplicate_sids)
                if len(sids) == 20:
                    sids += '...'
                sids = ', '.join(sids)
                source_name = ', '.join(sorted(duplicate_source))

                raise ValidationError('The source contains conflicting SID (%s) with other sources (%s)' % (sids, source_name))

            if len(rules_update["added"]):
                Rule.objects.bulk_create(rules_update["added"])
            if len(rules_groups):
                for rule in rules_groups:
                    # If IP list is empty it will be deleted because it has not
                    # been put in a changed or unchanged list. So we just care
                    # about saving the rule.
                    if len(rules_groups[rule].ips_list) > 0:
                        rules_groups[rule].group_ips_list = ",".join(rules_groups[rule].ips_list)
                        rules_groups[rule].rev = rules_groups[rule].next_rev
                        rules_groups[rule].save()
            if len(flowbits["added"]["flowbit"]):
                Flowbit.objects.bulk_create(flowbits["added"]["flowbit"])
            if len(flowbits["added"]["through_set"]):
                Flowbit.set.through.objects.bulk_create(flowbits["added"]["through_set"])
            if len(flowbits["added"]["through_isset"]):
                Flowbit.isset.through.objects.bulk_create(flowbits["added"]["through_isset"])
            rules_update["deleted"] = list(
                set(rules_list) -
                set(rules_update["added"]).union(set(rules_update["updated"])) -
                set(rules_unchanged)
            )
            source.aggregate_update(rules_update)
        rfile.close()


    def create_update(self):
        # for each set
        update = {}
        update["deleted"] = self.json_rules_list(self.updated_rules["deleted"])
        update["added"] = self.json_rules_list(self.updated_rules["added"])
        update["updated"] = self.json_rules_list(self.updated_rules["updated"])
        repo = self.get_git_repo(delete=False)
        sha = repo.heads.master.log()[-1].newhexsha
        # 最后通过SourceUpdate
        SourceUpdate.objects.create(
            source=self,
            created_date=timezone.now(),
            data=json.dumps(update),
            version=sha,
            changed=len(update["deleted"]) + len(update["added"]) + len(update["updated"]),
        )


    def json_rules_list(self, rlist):
        rules = []
        for rule in rlist:
            rules.append({
                "sid": rule.sid,
                "msg": rule.msg,
                "category": rule.category.name,
                "pk": rule.pk}
            )
        # for each rule we create a json object sid + msg + content
        return rules


    #该函数回去获取public_source,并写到tmp文件里
    def update_ruleset_http(self, f):
        proxy_params = get_system_settings().get_proxy_params() if self.use_sys_proxy else None
        hdrs = {'User-Agent': 'scirius'}
        if self.authkey:
            hdrs['Authorization'] = self.authkey

        version_uri = None
        if self.uri.startswith('https://rules.emergingthreatspro.com/') or \
                self.uri.startswith('https://rules.emergingthreats.net/') or \
                self.uri.startswith('https://ti.stamus-networks.io/') or \
                self.datatype not in ('sigs', 'sig', 'other'):
            version_uri = os.path.join(os.path.dirname(self.uri), 'version.txt')

        try:
            version_server = 1
            if version_uri:
                resp = requests.get(version_uri, proxies=proxy_params, headers=hdrs, verify=self.cert_verif)
                resp.raise_for_status()
                version_server = int(resp.content.strip())

                if self.version < version_server:
                    version_uri = None

            if version_uri is None:
                resp = requests.get(self.uri, proxies=proxy_params, headers=hdrs, verify=self.cert_verif)
                resp.raise_for_status()
                
                f.write(resp.content)

                if self.version < version_server:
                    self.version = version_server

                return True

        except requests.exceptions.ConnectionError as e:
            if "Name or service not known" in str(e):
                raise IOError("Failure to resolve hostname, please check DNS configuration")
            elif "Connection timed out" in str(e):
                raise IOError("Connection error 'Connection timed out'")
            else:
                raise IOError("Connection error '%s'" % (e))
        except requests.exceptions.HTTPError:
            if resp.status_code == 404:
                raise IOError("URL not found on server (error 404), please check URL")
            raise IOError("HTTP error %d (%s) sent by server, please check URL or server" % (resp.status_code, resp.reason))
        except requests.exceptions.Timeout:
            raise IOError("Request timeout, server may be down")
        except requests.exceptions.TooManyRedirects:
            raise IOError("Too many redirects, server may be broken")
        return False


class SourceAtVersion(models.Model):
    def test(self):
        rule_buffer = self.to_buffer()
        # 把所有rule传递给test_rule_buffer判断rule是否合理
        return self.test_rule_buffer(rule_buffer)
    # to_buffer会把Source里面所有的rule给读取出来
    def to_buffer(self):
        categories = Category.objects.filter(source=self.source)
        rules = Rule.objects.filter(category__in=categories)
        file_content = "# Rules file for %s generated by Scirius at %s\n" % (self.name, str(timezone.now()))
        rules_content = [rule.content for rule in rules]
        file_content += "\n".join(rules_content)
        return file_content
    def test_rule_buffer(self, rule_buffer, single=False):
        # 创建一个tests_rules对象,详细代码见tests_rules.py
        testor = TestRules()
        tmpdir = tempfile.mkdtemp()
        cats_content, iprep_content = self.export_files(tmpdir)
        related_files = {}

        for root, _, files in os.walk(tmpdir):
            for f in files:
                fullpath = os.path.join(root, f)
                if os.path.getsize(fullpath) < 50 * 1024:
                    with open(fullpath, 'r') as cf:
                        related_files[f] = cf.read()
        shutil.rmtree(tmpdir)
        # 调用check_rule_buffer()函数
        return testor.check_rule_buffer(
            rule_buffer,
            related_files=related_files,
            single=single,
            cats_content=cats_content,
            iprep_content=iprep_content
        )

add_public_source.html页面代码及分析如下

<!-- 获取source.yaml里面的source和参数 -->
              {% for source, params in sources.items %}
              <div class="list-group-item">
                  <div class="list-group-item-header">
                      <div class="list-view-pf-expand">
                          <span class="fa fa-angle-right"></span>
                      </div>
                      <div class="list-view-pf-actions">
                      <!-- 如果没有设置了added标签,则设置button为enable,如果设置了added则设置button为disabled -->
                          {% if not params.added %}
                              {% if request.user.is_staff %}
                          <a class="add_source_button" name="{{ source }}" style="cursor: pointer;"><button class="btn btn-default">Enable</button></a>
                              {% else %}
                                <button class="btn btn-default" disabled>Available</button>
                              {% endif %}
                          {% else %}
                          <button class="btn btn-default" disabled>Enabled</button>
                          {% endif %}
                      </div>
                      <div class="list-view-pf-main-info">
                          <div class="list-view-pf-left">
                              <span class="fa fa-external-link list-view-pf-icon-sm"></span>
                          </div>
                          <div class="list-view-pf-body">
                              <div class="list-view-pf-description">
                                  <div class="list-group-item-heading">
                                      {{ source }}
                                  </div>
                                  <div class="list-group-item-text">
                                      {{ params.summary }}
                                  </div>
                              </div>
                              <div class="list-view-pf-additional-info">
                                  <div class="list-view-pf-additional-info-item"> <span class="fa fa-list-alt"> </span> License: {{ params.license }}
                                  </div>
                                  <div class="list-view-pf-additional-info-item">
                                      <span class="fa fa-shield"> </span> Vendor: {{ params.vendor }}
                                  </div>
                              </div>
                          </div>
                      </div>
                  </div>

                  <div class="list-group-item-container container-fluid hidden">
                      <div class="close">
                          <span class="pficon pficon-close"></span>
                      </div>
                      <div class="row">
                          <div class="col-md-6">
                              <dl class="dl-horizontal">
                                  {% if params.description %}
                                  <dt>Description</dt><dd>{{ params.description }}</dd>
                                  {% endif %}
                                  <dt>Source URL</dt><dd>{{ params.url }}</a></dd>
                                  {% if params.subscribe_url %}
                                  <dt>Subscribe URL</dt><dd><a href="{{ params.subscribe_url_cleaned }}">{{ params.subscribe_url }}</a></dd>
                                  {% endif %}
                                  {% if params.support_url %}
                                  <dt>Support URL</dt><dd><a href="{{ params.support_url_cleaned }}">{{ params.support_url }}</a></dd>
                                  {% endif %}
                              </dl>
                          </div>
                      </div>
                  </div>
              </div>

import_and_add_source.html

<script>
$( 'document' ).ready(function() {
        <!-- 如果update为true则调用update_activate_source -->
        {% if update %}
            {% if not rulesets %}
            update_activate_source({{ source.pk }}, null);
            {% else %}
            update_activate_source({{ source.pk }}, [ {{ rulesets|join:"," }} ], [ {{ ruleset_list|safeseq|join:"," }} ])
            {% endif %}

            window.addEventListener("beforeunload", function (e) {
              if (!warn_on_exit) {
                  return;
              }
              var confirmationMessage = "Warning, leaving page will interrupt source addition mechanism.";

              e.returnValue = confirmationMessage;     // Gecko, Trident, Chrome 34+
              return confirmationMessage;              // Gecko, WebKit, Chrome <34
            });
        {% endif %}
});

function activate_ruleset_from(src_pk, rulesets, ruleset_list, r_length)
{
   if (rulesets.length == 0) {
        $('#source_progress').width("100%");
        $('#source_progress').addClass("progress-bar-success");
        $('#source_progress').removeClass("progress-bar-danger");
        $('#source_progress').text("Source fully activated.");

        $('#init_details').append("<p><a href='{{ source.get_absolute_url }}'>See details of {{ source.name }} source.</a></p>");
        warn_on_exit = false;
        return;
   }
   ri = rulesets.pop()
   ruleset = ruleset_list.pop()
   warn_on_exit = true;
   var tgturl = "/rules/source/" + src_pk + "/activate/" + ri;
   $.ajax({
       url: tgturl,
       type: 'POST',
          success: function(data) {
             if (data == true) {
                 var progress = 80 + (r_length - rulesets.length) * 20 / rulesets.length;
                 $('#source_progress').width(progress + "%");
                 $('#source_progress').text("Source activated in " + ruleset + ".");
                 $("#init_details").append('<p class="text-success"> <span class="glyphicon glyphicon-ok"></span> Source activated in "' + ruleset + '"</p>');
                 activate_ruleset_from(src_pk, rulesets, ruleset_list, r_length);
             } else {
                 $('#source_progress').addClass("progress-bar-danger");
                 $('#source_progress').text("Could not activate source for '" + ruleset + "'.");
                 $('#init_details').append("<p class='text-danger'> <span class='glyphicon glyphicon-remove'></span> Could not activate source for '" + ruleset + "'.</p>");
                 warn_on_exit = false;
             }
          },
      error: function(data) {
             $('#source_progress').addClass("progress-bar-danger");
             $('#source_progress').text("Unable activate source in '" + ruleset + "'.");
             var error_actions = '<a href="{{ source.get_absolute_url }}delete"><button class="btn btn-primary" type="submit"><span class="glyphicon glyphicon-trash"> Delete source</span></button></a>'
		        error_actions += ' <a href="{{ source.get_absolute_url }}"><button class="btn btn-warning" id="continue" type="submit"><span class="glyphicon glyphicon-ok"> Ignore errors and continue</span></button></a>';
		        $("#init_details").append('<div id="error_actions">' + error_actions + '<div>');
             warn_on_exit = false;
      }
    });
}

function update_activate_source(src_pk, rulesets, ruleset_list)
{
    var tgturl = "/rules/source/" + src_pk + "/update";

    $('#source_progress').text("Updating source.");
    <!-- 通过ajax访问"/rules/source/" + src_pk + "/update" -->
    $.ajax({
       type:"POST",
       url: tgturl,
          <!-- 如果后端返回的data['status']为true -->   
          success: function(data) {
            if (data['status'] == true) { 
                $("#init_details").append('<p class="text-success"> <span class="glyphicon glyphicon-ok"></span></span> Source updated</p>');
                $('#source_progress').width("70%");
                <!-- 如果成功获取到规则文件,会调用test_source去判断是否可以正常加载且和其他规则匹配 -->
		        test_source(src_pk, rulesets, ruleset_list);
            <!-- 否则定义error_action -->
            } else {
                $('#source_progress').addClass("progress-bar-danger");
                $('#source_progress').text("Could not test source.");
                <!-- 把data['error']放到text-danger标签里 -->
                $("#init_details").append('<p class="text-danger"> <span class="glyphicon glyphicon-remove"></span> Error during source update: ' + data['errors'] + '</p>');
                var error_actions = '<a href="{{ source.get_absolute_url }}delete"><button class="btn btn-primary" type="submit"><span class="glyphicon glyphicon-trash"> Delete source</span></button></a>'
		        error_actions += ' <a href="{{ source.get_absolute_url }}"><button class="btn btn-warning" id="continue" type="submit"><span class="glyphicon glyphicon-ok"> Ignore errors and continue</span></button></a>';
		        $("#init_details").append('<div id="error_actions">' + error_actions + '<div>');
                warn_on_exit = false;
            }
          },
	  error: function(data) {
             $('#source_progress').addClass("progress-bar-danger");
             $('#source_progress').text("Unable to update source.");
             var err_str = 'Error during source update';
             if (data.statusText && data.statusText != 'error') {
                 err_str += ' (' + data.statusText + ')';
             }
             $("#init_details").append('<p class="text-danger"> <span class="glyphicon glyphicon-remove"> ' + err_str + '</span> </p>');
             var error_actions = '<a href="{{ source.get_absolute_url }}delete"><button class="btn btn-primary" type="submit"><span class="glyphicon glyphicon-trash"> Delete source</span></button></a>'
		        error_actions += ' <a href="{{ source.get_absolute_url }}"><button class="btn btn-warning" id="continue" type="submit"><span class="glyphicon glyphicon-ok"> Ignore errors and continue</span></button></a>';
		        $("#init_details").append('<div id="error_actions">' + error_actions + '<div>');
             warn_on_exit = false;
	  },
      timeout: 240 * 1000
    });
}

function test_source(src_pk, rulesets, ruleset_list)
{
    var tgturl = "/rules/source/" + src_pk + "/test";

    $('#source_progress').text("Testing source.");
    $.ajax({
       url: tgturl,
          success: function(data) {
            if (data['status'] == true) { 
                $("#init_details").append('<p class="text-success"> <span class="glyphicon glyphicon-ok"></span></span> Source is valid</p>');
                $('#source_progress').width("80%");
                if (! rulesets) {
                    rulesets = []
                }
                if ('warnings' in data && data['warnings'][0] != undefined) {
                    var warning_content = "";
                    for (i = 0; i < data['warnings'].length; i++) {
                            warning_content += '<li><span class="text-warning">' + data['warnings'][i]['message'] + '</span></li>';
                    }
                    $("#init_details").append('<p class="text-warning"> <span class="glyphicon glyphicon-ok"></span> Source test warnings: <ul>' + warning_content + '</ul></p>');
                }
                activate_ruleset_from(src_pk, rulesets, ruleset_list, rulesets.length)
            } else {
                $('#source_progress').addClass("progress-bar-danger");
                $('#source_progress').text("Source has errors.");
		var error_content = "";
                if (data['errors'][0] != undefined) {
                    for (i = 0; i < data['errors'].length; i++) {
                            error_content += '<li><span class="text-danger"><strong>' + data['errors'][i]['error'] + '</strong></span>: <span>' + data['errors'][i]['message'] + '</span></li>';
                    }
                } else {
                    if (data['errors']['message'] != undefined) {
                        error_content = data['errors']['message'];
                    } else {
                        error_content = "Unknown error";
                    }
                }
                $("#init_details").append('<p class="text-danger"> <span class="glyphicon glyphicon-remove"></span> Source test failure: <ul>' + error_content + '</ul></p>');
                if ('warnings' in data && data['warnings'][0] != undefined) {
                    var warning_content = "";
                    for (i = 0; i < data['warnings'].length; i++) {
                            warning_content += '<li><span class="text-warning">' + data['warnings'][i]['message'] + '</span></li>';
                    }
                    $("#init_details").append('<p class="text-warning"> <span class="glyphicon glyphicon-ok"></span> Source test warnings: <ul>' + warning_content + '</ul></p>');
                }
		var error_actions = '<a href="{{ source.get_absolute_url }}delete"><button class="btn btn-primary" type="submit"><span class="glyphicon glyphicon-trash"> Delete source</span></button></a>'
		error_actions += ' <button class="btn btn-warning" id="continue" type="submit"> <span class="glyphicon glyphicon-ok"> Ignore errors and continue</span></button>';
		$("#init_details").append('<div id="error_actions">' + error_actions + '</div>');
                warn_on_exit = false;
		$("#continue").click( function(event) {
                        $("#error_actions").slideUp();
                        if (! rulesets) {
                                rulesets = []
                        }
                        activate_ruleset_from(src_pk, rulesets, ruleset_list, rulesets.length)
		});
            }
          },
	  error: function(data) {
             $('#source_progress').addClass("progress-bar-danger");
             $('#source_progress').text("Unable to test source.");
             $("#init_details").append('<p class="text-danger"> <span class="glyphicon glyphicon-remove"></span> Error during source testing : ' + data.statusText + '</p>');
             var error_actions = '<a href="{{ source.get_absolute_url }}delete"><button class="btn btn-primary" type="submit"><span class="glyphicon glyphicon-trash"> Delete source</span></button></a>'
		        error_actions += ' <a href="{{ source.get_absolute_url }}"><button class="btn btn-warning" id="continue" type="submit"><span class="glyphicon glyphicon-ok"> Ignore errors and continue</span></button></a>';
		        $("#init_details").append('<div id="error_actions">' + error_actions + '<div>');
             warn_on_exit = false;
	  },
      timeout: 1200 * 1000
    });

}
</script>